#!/usr/bin/python

import xml.etree.cElementTree as ET
from urllib import quote_plus
import sys
from glob import glob
import traceback
import re

# optimizations
try:
    import psyco
    psyco.full()
except:
    print "No runtime optimization."
    pass

# templates
DEFAULT_OUTPUT="output.turtle"
TEMPLATE_DIR="templates"
SOFTWARE_NAMESPACE="http://mephistopedia.net/"
PACKAGE_NAMESPACE="http://mephisto.mandriva.com/"

templates = {}
for t in ["header", "bug", "reporter", "package", "comment", "resolution", "software", "cc", "blocks", "assigned_to"]:
    templates[t] = open("%s/%s" % (TEMPLATE_DIR, t)).read()

rpm_r = re.compile("(.*?)-(\d.*)-.*")

datetime_r = re.compile("(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):?(\d\d)?")
def parse_ts(s):
    """Parses timestamp and outputs valid xsd:dateTime"""
    if len(s) < 16:
        # too short, probably an invalid timestamp
        print >>sys.stderr, "Invalid datetime: %s" % s
        return s
    try:
        year, month, day, hour, min, sec = datetime_r.findall(s)[0]
    except:
        traceback.print_exc()
        return s
    if len(sec) < 1:
        sec = "00"
    # a valid xsd:dateTime
    ts = "%s-%s-%sT%s:%s:%s" % (year, month, day, hour, min, sec)
    return ts

# character escaping
def escapechars(s, chars=[ '\"', '\'', '\\' ]):
    """Converts invalid characters to corresponding html"""
    return "".join(['\\%s' % c if c in chars else c for c in s])

# xml fixing
# from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML
RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
        u'|' + \
        u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \
        (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
                unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
                unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff))
regex = re.compile(RE_XML_ILLEGAL)

def parse_bugs(file, output=sys.stdout):
    """Parses bugzilla bugs and outputs valid turtle format"""
    data = open(file).read()

    try:
        res = ET.fromstring(data)
    except SyntaxError:
        # looks like we found an invalid character
        print >>sys.stderr, "Invalid characters found, stripping..",
        # this is slow, but it is the best we can do right now..
        for match in regex.finditer(data):
            data = data[:match.start()] + "?" + data[match.end():]
        res = ET.fromstring(data)

    baseurl = res.get("urlbase")
    trail = baseurl[len(baseurl)-1]
    slash = "/"
    if trail != slash:
        baseurl = baseurl + "/"
    print "\nBase URL: %s" % baseurl
    print >> output, templates["header"]
    for bug in res:
        # Check if bug is valid
        if bug.get("error"):
            # there is some error with this bug
            # TODO: do we actually need the information about this error?
            # usually the error fields means that this bug does not exists.
            continue
        # XPATH
        params = {}
        # process bug parameters
        bug_id = bug.findtext("./bug_id")
        url = "%s%s" % (baseurl, bug_id)
        bugzilla_url = "%sshow_bug.cgi?id=%s" % (baseurl, bug_id)
        params["bug_id"] = bug_id
        params["url"] = url
        params["bugzilla_url"] = bugzilla_url
        short_desc = bug.findtext("./short_desc", "Bug %s (no short description)" % bug_id)
        params["short_desc"] = escapechars(short_desc.encode("utf-8"))
        params["creation_ts"] = parse_ts(bug.findtext("./creation_ts"))
        params["sys"] = bug.findtext("./sys")
        params["bug_status"] = bug.findtext("./bug_status").capitalize()
        params["keywords"] = bug.findtext("./keywords")
        params["delta_ts"] = bug.findtext("./delta_ts")
        params["namespace"] = SOFTWARE_NAMESPACE
        bug_priority = bug.findtext("./priority")
        if bug_priority == "release_critical":
            priority = 1
        elif bug_priority == "high":
            priority = 2
        elif bug_priority == "normal":
            priority = 3
        elif bug_priority == "low":
            priority = 4
        else:
            priority = 5
        params["priority"] = priority
        # TODO: severity!
        severity = bug.findtext("./severity")
        if severity:
            params["severity"] = severity.capitalize()
        else:
            params["severity"] = "Normal"
        description = bug.findtext("./long_desc/thetext", "")
        params["description"] = escapechars(description.encode("utf-8"))
        print >>output, templates["bug"] % params
        
        #assignee
        assigned_to = bug.find("./assigned_to")
        if assigned_to.text:
            fullname = assigned_to.get("name")
            #print "fullname: %s " % fullname
            #fullname = escapechars(fullname.encode("utf-8"))
            print >>output, templates["assigned_to"] % {
                            "url": url,
                            "assigned_to": assigned_to.text,
                            "namespace": SOFTWARE_NAMESPACE
                            }
        #blocks
        for blocked in bug.findall("./blocked"):
            if blocked.text:
                print >>output, templates["blocks"] % {
                                "url": url,
                                "blocked": "%s%s" % (baseurl, blocked.text)
                                }
        
        #cc
        for cc in bug.findall("./cc"):
            if cc.text:
                print >>output, templates["cc"] % {
                                "url": url,
                                "cc": cc.text,
                                "namespace": SOFTWARE_NAMESPACE
                            }
        

        # software, version and rpm
        product = bug.findtext("./product", None)
        if product:
            quoted_product = quote_plus(product)
            version = bug.findtext("./version", "")
            print >>output, templates["software" ] % {
                            "url": url,
                            "quoted_product": quoted_product,
                            "product": product,
                            "version": version,
                            "namespace": SOFTWARE_NAMESPACE
                            }
        package = bug.findtext("./cf_rpmpkg", None)
        if package:
            ret = rpm_r.findall(package)
            if ret:
                try:
                    product, version = ret[0]
                    quoted_product = quote_plus(product)
                    quoted_package = quote_plus(package)
                    print >>output, templates["software"] % {
                            "url": url,
                            "quoted_product": quoted_product,
                            "product": product,
                            "version": version,
                            "namespace": SOFTWARE_NAMESPACE
                            }
                    print >>output, templates["package"] % {
                            "quoted_product": quoted_product,
                            "package": package,
                            "quoted_package": quoted_package,
                            "namespace": PACKAGE_NAMESPACE
                            }
                except:
                    print >>sys.stderr, "error parsing rpm resource: %s" % package,

        # reporter
        reporter = bug.findtext("./reporter")
        quoted_email = reporter.replace(" ", "+")
        print >>output, templates["reporter"] % {
                            "url": url,
                            "email": reporter,
                            "quoted_email": quoted_email,
                            "namespace": SOFTWARE_NAMESPACE
                        }

        # resolution
        resolution = bug.findtext("./resolution")
        if resolution:
            # convert resolution
            resolution = resolution.upper()
            if resolution == "WONTFIX":
                resolution_s = "WontFix"
            elif resolution == "WORKSFORME":
                resolution_s = "WorksForMe"
            elif resolution == "THIRDPARTY":
                resolution_s = "ThirdParty"
            else:
                resolution_s = resolution.capitalize()

            print >>output, templates["resolution"] % {
                            "url": url,
                            "resolution": resolution_s
                            }

        # now let's look at comments
        comment_id = 0
        for comment in bug.findall("./long_desc"):
            who = comment.find("./who")
            nick = quote_plus(who.get("name").encode('utf-8'))
            email = who.text
            quoted_email = email.replace(" ", "+") # remove blank spaces from emails
            comment_ts = parse_ts(comment.findtext("./bug_when"))
            description = escapechars(comment.findtext("./thetext").encode("utf-8"))
            print >>output, templates["comment"] % {
                            "url": url,
                            "comment_id": comment_id,
                            "email": email,
                            "quoted_email": quoted_email,
                            "nick": nick,
                            "description": description,
                            "comment_ts": comment_ts,
                            "namespace": SOFTWARE_NAMESPACE
                            }
            comment_id += 1

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print "Usage: %s <bugs dir> [output file]" % sys.argv[0]
        sys.exit(1)
    bugsdir = sys.argv[1]
    if len(sys.argv) == 3:
        output = open(sys.argv[2], "w")
        one_big_file = True
    else:
        output = None
        one_big_file = False

    # looking for files
    files = glob("%s/*xml" % bugsdir)
    for file in files:
        # are we doing one file at a time?
        if not one_big_file:
            newfile_l = file.split(".")
            newfile_l[-1] = "turtle"
            newfile = ".".join(newfile_l)
            output = open(newfile, "w")
        print >>sys.stderr, "Parsing %s.." % file,
        try:
            parse_bugs(file, output=output)
            print >>sys.stderr, "ok"
#            os.unlink(file)
        except:
            traceback.print_exc()
            print >>sys.stderr, "error parsing %s: %s" % (file, sys.exc_value)
            sys.exit(1)
        if not one_big_file:
            output.close()
#    print files
